Reproducible Documentation of Analysis Study 1

R Version

R.Version()
$platform
[1] "aarch64-apple-darwin20"

$arch
[1] "aarch64"

$os
[1] "darwin20"

$system
[1] "aarch64, darwin20"

$status
[1] ""

$major
[1] "4"

$minor
[1] "2.2"

$year
[1] "2022"

$month
[1] "10"

$day
[1] "31"

$`svn rev`
[1] "83211"

$language
[1] "R"

$version.string
[1] "R version 4.2.2 (2022-10-31)"

$nickname
[1] "Innocent and Trusting"

Import of the Data

The data was assessed with the formr survey framework (Arslan, Walther, and Tata 2020). The raw data was imported via the following code.

library(tidyverse)
library(ggforce)
library(lme4)
library(BFpack)
library(hrbrthemes)
library(patchwork)
library(brms)
library(viridis)
library(ggdist)
library(tidybayes)
library(here)
library(skimr)
load(here("data/teachers_study1_N40.RData"))

skim(study1)

set.seed(25051982)

Data Wrangling

# wrangle information on the plot type, ES, ...
plot_info <- study1 %>%
    pivot_longer(2:195, names_to = "variables", values_to = "values", 
                 values_transform = as.character) %>%
    dplyr::filter(str_detect(variables, "plot")) %>% 
    # we only need the rows with info on plots
    tidyr::separate(col = values, into = c("type", "axis", "effsize"), 
                    # separate the info into three columns
                    sep = "_", remove = F) %>%
    dplyr::mutate(plot = variables,       # rename variables for later join
                  type = paste(type, axis, sep = "_")) %>%
    dplyr::select(-variables, -axis)

# wrangle answers to items on each page
item_values <- study1 %>%
    dplyr::select(-c(topic:itemo)) %>%
    pivot_longer(2:169, names_to = "variables", values_to = "values", 
                 values_transform = as.character) %>%
    dplyr::mutate(variables = case_when(      # recode variable names that have
        variables == "sensi_6" ~ "sensi_06",  # accidentally been labeled
        variables == "acccl_6" ~ "acccl_06",  # without zero
        variables == "accu3_6" ~ "accu3_06",
        variables == "accov_6" ~ "accov_06",
        variables == "diffi_6" ~ "diffi_06",
        variables == "infor_6" ~ "infor_06",
        variables == "value_6" ~ "value_06",
        TRUE ~ variables 
    )) %>%
    dplyr::mutate(plot = paste0("plotx_", str_sub(variables, -2, -1)), 
                  # create variable for later join
                  variables = str_sub(variables, 1, -4)) %>%    
    # rename variable names to get a data set 
    # with one line per participant per page
    pivot_wider(id_cols = c(session, plot), names_from = "variables", 
                values_from = "values")

skim(plot_info)
Data summary
Name plot_info
Number of rows 960
Number of columns 5
_______________________
Column type frequency:
character 5
________________________
Group variables None

Variable type: character

skim_variable n_missing complete_rate min max empty n_unique whitespace
session 0 1 64 64 0 40 0
values 0 1 17 24 0 24 0
type 0 1 13 19 0 4 0
effsize 0 1 3 4 0 6 0
plot 0 1 8 8 0 24 0
skim(item_values)
Data summary
Name item_values
Number of rows 960
Number of columns 9
_______________________
Column type frequency:
character 9
________________________
Group variables None

Variable type: character

skim_variable n_missing complete_rate min max empty n_unique whitespace
session 0 1.0 64 64 0 40 0
plot 0 1.0 8 8 0 24 0
sensi 480 0.5 59 62 0 3 0
acccl 480 0.5 1 4 0 11 0
accu3 480 0.5 1 3 0 48 0
accov 480 0.5 1 3 0 52 0
diffi 0 1.0 1 1 0 7 0
infor 0 1.0 1 1 0 7 0
value 0 1.0 1 1 0 7 0
# join the two data sets
study1_w <- full_join(plot_info, item_values, 
                               by = c("session", "plot")) %>% 
    # by participant and page (plot)
    dplyr::select(-values) %>%
    dplyr::mutate(rating_cl = as.numeric(acccl), # some var need to be defined as
                  rating_u3 = as.numeric(accu3), # numeric again
                  rating_ov = as.numeric(accov),
                  diffi = as.numeric(diffi),
                  infor = as.numeric(infor),
                  value = as.numeric(value),
                  effsize = as.numeric(effsize),
                  effsize_cl = case_when( 
                  # there is no negative Cliff's Delta, so we have to compute 
                  # two transformations
                      effsize > 0 ~   (((2*pnorm(effsize/2))-1)/pnorm(effsize/2)),
                  # transform the actual effect size Cohen's d to Cliff's Delta
                      effsize < 0 ~ (- (((2*pnorm(abs(effsize)/2))-1)/pnorm(abs(effsize)/2))) 
                  # transform the actual effect size Cohen's d to Cliff's Delta 
                  # and make it negative as in the item
                  ),
                  effsize_u3 = 1-pnorm(effsize), # reverse so that it fits the direction of the U3 item
                  # transform the actual effect size Cohen's d to Cohen's U3
                  effsize_ov = 2 * pnorm(-abs(effsize) / 2), 
                  # transform the actual effect size Cohen's d to overlap
                  # actual difference of rating relative to depicted effectsize 
                  diff_cl = (rating_cl - effsize_cl)/2,
                  # actual difference of rating relative to depicted effectsize
                  diff_u3 = (rating_u3/100) - effsize_u3,
                  # actual difference of rating relative to depicted effectsize 
                  diff_ov = (rating_ov/100) - effsize_ov,
                  diffi_normed = ((diffi - 1)  / 3) - 1, # transform item to -1 to 1
                  infor_normed = ((infor - 1)  / 3) - 1, # transform item to -1 to 1
                  value_normed = ((value - 1)  / 3) - 1) %>%  # transform item to -1 to 1
    group_by(session) %>% 
    mutate(rating_ov_missconcept = median(rating_ov, na.rm = T) < 68.9,
           rating_u3_missconcept = median(rating_u3, na.rm = T) < 21.2) %>% 
    ungroup() %>% 
    mutate(rating_u3_filtered = ifelse(rating_u3_missconcept == T, NA, rating_u3),
           rating_ov_filtered = ifelse(rating_ov_missconcept == T, NA, rating_ov),
           diff_u3_filtered = (rating_u3_filtered/100) - effsize_u3,
           diff_ov_filtered = (rating_ov_filtered/100) - effsize_ov,
           sensi_binary = ifelse(is.na(sensi), # 1 if NOT "equal"
                                        NA,
                                        as.numeric(!grepl("equal", sensi))),
           sensi_ordinal = ordered(factor(substr(sensi, 55, 100)),
                                   levels = c("inferior",
                                              "equal",
                                              "superior")),
           sensi_binary_filtered = case_when(sensi_ordinal == "equal" ~ 0, 
                                             (sensi_ordinal == "inferior" & 
                                                  effsize < 0) | 
                                                  (sensi_ordinal == "superior" & 
                                                  effsize > 0) ~ as.numeric(NA),
                                             (sensi_ordinal == "inferior" & # was not there
                                                  effsize > 0) | 
                                                  (sensi_ordinal == "superior" & 
                                                  effsize < 0) ~ 1, 
                                             TRUE ~ as.numeric(NA)), # was 1
           sensi_correct = case_when(sensi_ordinal == "equal" ~ "judged equal", 
                                     (sensi_ordinal == "inferior" & 
                                                  effsize < 0) | 
                                                  (sensi_ordinal == "superior" & 
                                                  effsize > 0) ~ "wrong direction",
                                             (sensi_ordinal == "inferior" & # was not there
                                                  effsize > 0) | 
                                                  (sensi_ordinal == "superior" & 
                                                  effsize < 0) ~ "right direction", 
                                             TRUE ~ NA_character_),
           effsize_abs = abs(effsize))

skim(study1_w)
Data summary
Name study1_w
Number of rows 960
Number of columns 34
_______________________
Column type frequency:
character 8
factor 1
logical 2
numeric 23
________________________
Group variables None

Variable type: character

skim_variable n_missing complete_rate min max empty n_unique whitespace
session 0 1.0 64 64 0 40 0
type 0 1.0 13 19 0 4 0
plot 0 1.0 8 8 0 24 0
sensi 480 0.5 59 62 0 3 0
acccl 480 0.5 1 4 0 11 0
accu3 480 0.5 1 3 0 48 0
accov 480 0.5 1 3 0 52 0
sensi_correct 480 0.5 12 15 0 3 0

Variable type: factor

skim_variable n_missing complete_rate ordered n_unique top_counts
sensi_ordinal 480 0.5 TRUE 3 sup: 177, inf: 159, equ: 144

Variable type: logical

skim_variable n_missing complete_rate mean count
rating_ov_missconcept 0 1 0.22 FAL: 744, TRU: 216
rating_u3_missconcept 0 1 0.55 TRU: 528, FAL: 432

Variable type: numeric

skim_variable n_missing complete_rate mean sd p0 p25 p50 p75 p100 hist
effsize 0 1.00 0.00 0.56 -0.80 -0.50 0.00 0.50 0.80 ▇▃▁▃▇
diffi 0 1.00 4.05 1.71 1.00 3.00 4.00 5.00 7.00 ▇▆▆▆▇
infor 0 1.00 4.11 1.45 1.00 3.00 4.00 5.00 7.00 ▅▅▇▇▅
value 0 1.00 4.13 1.49 1.00 3.00 4.00 5.00 7.00 ▅▃▆▇▅
rating_cl 480 0.50 0.01 0.38 -1.00 -0.20 0.00 0.20 1.00 ▂▆▇▃▁
rating_u3 480 0.50 26.17 22.60 0.00 6.00 18.00 45.00 100.00 ▇▃▃▁▁
rating_ov 480 0.50 71.63 28.63 0.00 65.00 80.00 90.00 100.00 ▂▁▁▅▇
effsize_cl 0 1.00 0.00 0.34 -0.47 -0.33 0.00 0.33 0.47 ▇▃▁▃▇
effsize_u3 0 1.00 0.50 0.21 0.21 0.31 0.50 0.69 0.79 ▇▃▁▃▇
effsize_ov 0 1.00 0.80 0.09 0.69 0.69 0.80 0.92 0.92 ▇▁▇▁▇
diff_cl 480 0.50 0.01 0.18 -0.54 -0.06 0.00 0.07 0.64 ▁▂▇▂▁
diff_u3 480 0.50 -0.24 0.29 -0.79 -0.49 -0.21 -0.03 0.69 ▅▆▇▂▁
diff_ov 480 0.50 -0.09 0.29 -0.92 -0.12 0.01 0.10 0.31 ▂▁▁▇▅
diffi_normed 0 1.00 0.02 0.57 -1.00 -0.33 0.00 0.33 1.00 ▇▆▆▆▇
infor_normed 0 1.00 0.04 0.48 -1.00 -0.33 0.00 0.33 1.00 ▅▅▇▇▅
value_normed 0 1.00 0.04 0.50 -1.00 -0.33 0.00 0.33 1.00 ▅▃▆▇▅
rating_u3_filtered 744 0.22 43.06 17.90 0.00 30.00 45.00 55.00 80.00 ▂▃▇▇▂
rating_ov_filtered 588 0.39 82.60 16.81 0.00 80.00 85.00 90.00 100.00 ▁▁▁▅▇
diff_u3_filtered 744 0.22 -0.06 0.22 -0.79 -0.15 -0.03 0.06 0.54 ▁▂▇▅▁
diff_ov_filtered 588 0.39 0.02 0.17 -0.92 -0.02 0.05 0.10 0.31 ▁▁▁▇▅
sensi_binary 480 0.50 0.70 0.46 0.00 0.00 1.00 1.00 1.00 ▃▁▁▁▇
sensi_binary_filtered 507 0.47 0.68 0.47 0.00 0.00 1.00 1.00 1.00 ▃▁▁▁▇
effsize_abs 0 1.00 0.50 0.25 0.20 0.20 0.50 0.80 0.80 ▇▁▇▁▇
skim(study1_w %>%
         select(diffi,infor,value,rating_u3, rating_ov, rating_cl))
Data summary
Name %>%(…)
Number of rows 960
Number of columns 6
_______________________
Column type frequency:
numeric 6
________________________
Group variables None

Variable type: numeric

skim_variable n_missing complete_rate mean sd p0 p25 p50 p75 p100 hist
diffi 0 1.0 4.05 1.71 1 3.0 4 5.0 7 ▇▆▆▆▇
infor 0 1.0 4.11 1.45 1 3.0 4 5.0 7 ▅▅▇▇▅
value 0 1.0 4.13 1.49 1 3.0 4 5.0 7 ▅▃▆▇▅
rating_u3 480 0.5 26.17 22.60 0 6.0 18 45.0 100 ▇▃▃▁▁
rating_ov 480 0.5 71.63 28.63 0 65.0 80 90.0 100 ▂▁▁▅▇
rating_cl 480 0.5 0.01 0.38 -1 -0.2 0 0.2 1 ▂▆▇▃▁
sociodemographics <- read_delim("data/teachers_study1a.csv", delim = ";") %>%
    select(session, mcstu, texpe, mcsub) %>% 
    # filter(!is.na(mcstu & texpe)) %>%
    mutate(reply = session %in% c(study1_w$session)) %>%
    filter(!reply == "FALSE") %>%
    select(-reply) %>%
    mutate(mcstu = as.factor(mcstu),
    texpe = as.numeric(texpe),
    mcsub = as.factor(mcsub),
    subject_stem = grepl("1", mcsub),
    subject_languages = grepl("2", mcsub),
    subject_humanities_socialscience = grepl("3", mcsub),
    subject_asthetic = grepl("4", mcsub)) 

skim(sociodemographics) # n = 15 participants checked "others" when asked which school type they teach at
Data summary
Name sociodemographics
Number of rows 40
Number of columns 8
_______________________
Column type frequency:
character 1
factor 2
logical 4
numeric 1
________________________
Group variables None

Variable type: character

skim_variable n_missing complete_rate min max empty n_unique whitespace
session 0 1 64 64 0 40 0

Variable type: factor

skim_variable n_missing complete_rate ordered n_unique top_counts
mcstu 0 1 FALSE 6 6: 15, 1: 9, 2: 7, 3: 6
mcsub 0 1 FALSE 10 1: 8, 3: 8, 2: 7, 1, : 3

Variable type: logical

skim_variable n_missing complete_rate mean count
subject_stem 0 1 0.48 FAL: 21, TRU: 19
subject_languages 0 1 0.45 FAL: 22, TRU: 18
subject_humanities_socialscience 0 1 0.50 FAL: 20, TRU: 20
subject_asthetic 0 1 0.15 FAL: 34, TRU: 6

Variable type: numeric

skim_variable n_missing complete_rate mean sd p0 p25 p50 p75 p100 hist
texpe 0 1 10.15 8.66 2 4 7 13.5 37.5 ▇▃▂▁▁
study1_w_demo <- full_join(study1_w, sociodemographics, by = "session") %>%
    mutate(schooltype_binary = ifelse(mcstu == 6, 1, 0)) # 6 = others 

study1_w_type <- study1_w %>%
         select(diffi, infor,value,rating_u3, rating_ov, rating_cl, session, 
                type, plot) %>%
    gather(var, value, diffi, infor,value,rating_u3, rating_ov, rating_cl)%>%
    mutate(variable = paste(var, type, sep = "_"))%>%
    select(-type, -var)%>%
    group_by(session, plot) %>%
    spread(variable, value) %>%
    ungroup()
 
skim(study1_w_type)
Data summary
Name study1_w_type
Number of rows 960
Number of columns 26
_______________________
Column type frequency:
character 2
numeric 24
________________________
Group variables None

Variable type: character

skim_variable n_missing complete_rate min max empty n_unique whitespace
session 0 1 64 64 0 40 0
plot 0 1 8 8 0 24 0

Variable type: numeric

skim_variable n_missing complete_rate mean sd p0 p25 p50 p75 p100 hist
diffi_gardneraltman_xaxis 720 0.25 3.59 1.87 1 2.00 3.0 5.00 7.0 ▇▅▃▃▅
diffi_halfeye_xaxis 720 0.25 4.23 1.54 1 3.00 4.0 6.00 7.0 ▅▇▆▆▇
diffi_halfeye_yaxis 720 0.25 4.55 1.58 1 3.00 5.0 6.00 7.0 ▃▃▃▆▇
diffi_raincloud_yaxis 720 0.25 3.83 1.68 1 2.75 4.0 5.00 7.0 ▇▆▆▅▆
infor_gardneraltman_xaxis 720 0.25 3.80 1.57 1 3.00 4.0 5.00 7.0 ▆▅▆▇▃
infor_halfeye_xaxis 720 0.25 4.19 1.34 1 3.00 4.0 5.00 7.0 ▃▆▇▇▅
infor_halfeye_yaxis 720 0.25 4.40 1.35 1 3.00 4.5 5.00 7.0 ▂▅▇▇▆
infor_raincloud_yaxis 720 0.25 4.04 1.48 1 3.00 4.0 5.00 7.0 ▅▃▇▆▅
rating_cl_gardneraltman_xaxis 843 0.12 0.03 0.40 -1 -0.20 0.0 0.20 1.0 ▂▅▇▃▁
rating_cl_halfeye_xaxis 834 0.13 0.00 0.36 -1 -0.20 0.0 0.20 1.0 ▁▇▇▂▁
rating_cl_halfeye_yaxis 842 0.12 0.01 0.40 -1 -0.20 0.0 0.20 1.0 ▂▇▇▃▁
rating_cl_raincloud_yaxis 841 0.12 0.03 0.38 -1 -0.20 0.0 0.20 0.8 ▁▃▇▆▂
rating_ov_gardneraltman_xaxis 843 0.12 68.63 30.39 0 60.00 80.0 90.00 100.0 ▂▁▂▅▇
rating_ov_halfeye_xaxis 834 0.13 74.73 26.46 0 70.00 85.0 90.00 100.0 ▂▁▁▃▇
rating_ov_halfeye_yaxis 842 0.12 70.08 30.92 0 66.25 80.0 90.00 100.0 ▂▁▁▅▇
rating_ov_raincloud_yaxis 841 0.12 72.83 26.57 0 67.50 80.0 90.00 100.0 ▂▁▁▅▇
rating_u3_gardneraltman_xaxis 843 0.12 26.56 21.61 0 7.00 22.0 45.00 100.0 ▇▃▅▁▁
rating_u3_halfeye_xaxis 834 0.13 26.73 23.01 0 5.25 20.0 48.75 80.0 ▇▂▂▃▁
rating_u3_halfeye_yaxis 842 0.12 27.08 23.33 0 7.00 15.0 50.00 75.0 ▇▁▂▃▁
rating_u3_raincloud_yaxis 841 0.12 24.28 22.54 0 5.50 15.0 40.00 100.0 ▇▃▂▁▁
value_gardneraltman_xaxis 720 0.25 3.80 1.51 1 3.00 4.0 5.00 7.0 ▆▅▇▇▃
value_halfeye_xaxis 720 0.25 4.31 1.40 1 3.75 5.0 5.00 7.0 ▃▃▆▇▅
value_halfeye_yaxis 720 0.25 4.38 1.39 1 4.00 5.0 5.00 7.0 ▂▂▅▇▃
value_raincloud_yaxis 720 0.25 4.03 1.58 1 3.00 4.0 5.00 7.0 ▆▃▆▇▅
# abstract metric
mean(study1_w_type$rating_cl_gardneraltman_xaxis, na.rm=TRUE)
[1] 0.02905983
sd(study1_w_type$rating_cl_gardneraltman_xaxis, na.rm=TRUE)
[1] 0.3976352
mean(study1_w_type$rating_cl_halfeye_xaxis, na.rm=TRUE)
[1] -0.003174603
sd(study1_w_type$rating_cl_halfeye_xaxis, na.rm=TRUE)
[1] 0.3550632
mean(study1_w_type$rating_cl_halfeye_yaxis, na.rm=TRUE)
[1] 0.005084746
sd(study1_w_type$rating_cl_halfeye_yaxis, na.rm=TRUE)
[1] 0.402948
mean(study1_w_type$rating_cl_raincloud_yaxis, na.rm=TRUE)
[1] 0.0302521
sd(study1_w_type$rating_cl_raincloud_yaxis, na.rm=TRUE)
[1] 0.3805782
# u3
mean(study1_w_type$rating_u3_gardneraltman_xaxis, na.rm=TRUE)
[1] 26.5641
sd(study1_w_type$rating_u3_gardneraltman_xaxis, na.rm=TRUE)
[1] 21.60994
mean(study1_w_type$rating_u3_halfeye_xaxis, na.rm=TRUE)
[1] 26.73016
sd(study1_w_type$rating_u3_halfeye_xaxis, na.rm=TRUE)
[1] 23.01318
mean(study1_w_type$rating_u3_halfeye_yaxis, na.rm=TRUE)
[1] 27.08475
sd(study1_w_type$rating_u3_halfeye_yaxis, na.rm=TRUE)
[1] 23.33354
mean(study1_w_type$rating_u3_raincloud_yaxis, na.rm=TRUE)
[1] 24.27731
sd(study1_w_type$rating_u3_raincloud_yaxis, na.rm=TRUE)
[1] 22.53657
# overlap
mean(study1_w_type$rating_ov_gardneraltman_xaxis, na.rm=TRUE)
[1] 68.63248
sd(study1_w_type$rating_ov_gardneraltman_xaxis, na.rm=TRUE)
[1] 30.38648
mean(study1_w_type$rating_ov_halfeye_xaxis, na.rm=TRUE)
[1] 74.73016
sd(study1_w_type$rating_ov_halfeye_xaxis, na.rm=TRUE)
[1] 26.46217
mean(study1_w_type$rating_ov_halfeye_yaxis, na.rm=TRUE)
[1] 70.07627
sd(study1_w_type$rating_ov_halfeye_yaxis, na.rm=TRUE)
[1] 30.92095
mean(study1_w_type$rating_ov_raincloud_yaxis, na.rm=TRUE)
[1] 72.83193
sd(study1_w_type$rating_ov_raincloud_yaxis, na.rm=TRUE)
[1] 26.56693
# create a list of u3_misconceptualizers
u3_misconceptualizers <-
    study1_w %>% 
    filter(rating_u3_missconcept == T) %>% 
    pull(session) %>% 
    unique()

# create a list of ov_misconceptualizers
ov_misconceptualizers <-
    study1_w %>% 
    filter(rating_ov_missconcept == T) %>% 
    pull(session) %>% 
    unique()    

### wrangle time stamp data ####################################################
study1_w_timestamp <- 
    read_csv(here("data/teachers_study1_N40_detailed.csv")) %>% 
    # filter participants from study1_w only
    filter(session %in% study1_w$session) %>% 
    # we only need vars sensitivity or accuracy
    dplyr::filter(str_detect(item_name, "sensi|acccl|accu3|accov")) %>%  
    # create var with plot number
    mutate(plot = paste0("plotx_", str_sub(item_name, -2, -1)),
           # recode wrong item labelling
           plot = ifelse(plot == "plotx__6", "plotx_06", plot)) %>% 
    relocate(session, plot) %>% 
    # delete the page number in item name
    mutate(item_name = str_sub(item_name, 1, 5)) %>%  
    pivot_wider(id_cols = c(session, plot), names_from = item_name, 
                values_from = answered_relative) %>% 
    rowwise() %>%
    # what was the time of the first item to be clicked?
    mutate(effic = min(sensi, acccl, accu3, accov, na.rm=T)) %>%
    ungroup() %>% 
    dplyr::select(session, plot, effic, sensi, acccl, accu3, accov) %>% 
    left_join(., study1_w %>% 
                  select(session, plot, type), by=c("session", "plot")) %>% 
    # generate data set so that the six plots from the same type are ordered
    # one after the other (and not 1-24)
    group_by(session, type) %>% 
    arrange(plot) %>% 
    mutate(plotNrWithin = 1:n()) %>%
    ungroup() %>% 
    group_by(plotNrWithin, type) %>% 
    mutate(effic_10righttrunc = ifelse(effic > quantile(effic, .9), NA, effic),
           effic_05righttrunc = ifelse(effic > quantile(effic, .95), NA, effic),
           log_effic_05righttrunc =log(effic_05righttrunc),
           log_effic_10righttrunc = log(effic_10righttrunc),
           plotNrWithin0 = plotNrWithin -1,
           plotNrWithin_factor = as.factor(plotNrWithin)) %>% 
    ungroup()

skim(study1_w_timestamp)
Data summary
Name study1_w_timestamp
Number of rows 960
Number of columns 15
_______________________
Column type frequency:
character 3
factor 1
numeric 11
________________________
Group variables None

Variable type: character

skim_variable n_missing complete_rate min max empty n_unique whitespace
session 0 1 64 64 0 40 0
plot 0 1 8 8 0 24 0
type 0 1 13 19 0 4 0

Variable type: factor

skim_variable n_missing complete_rate ordered n_unique top_counts
plotNrWithin_factor 0 1 FALSE 6 1: 160, 2: 160, 3: 160, 4: 160

Variable type: numeric

skim_variable n_missing complete_rate mean sd p0 p25 p50 p75 p100 hist
effic 0 1.00 28817.81 67778.16 2419.00 9065.93 14866.60 26444.00 1342595.60 ▇▁▁▁▁
sensi 480 0.50 22664.69 53591.36 2419.00 7670.72 11408.65 20709.33 684967.00 ▇▁▁▁▁
acccl 557 0.42 33590.02 83104.51 3547.30 11650.00 18116.60 32103.65 1349401.70 ▇▁▁▁▁
accu3 480 0.50 61388.83 88359.00 5599.30 22174.12 39723.25 70041.55 1342595.60 ▇▁▁▁▁
accov 480 0.50 69364.58 90056.87 6901.30 26939.18 46882.25 79074.25 1344927.30 ▇▁▁▁▁
plotNrWithin 0 1.00 3.50 1.71 1.00 2.00 3.50 5.00 6.00 ▇▃▃▃▃
effic_10righttrunc 96 0.90 16914.40 11789.22 2419.00 8734.20 13402.60 21572.35 89854.90 ▇▂▁▁▁
effic_05righttrunc 48 0.95 19435.38 17815.29 2419.00 8910.87 14118.35 23782.15 168247.90 ▇▁▁▁▁
log_effic_05righttrunc 48 0.95 9.60 0.71 7.79 9.10 9.56 10.08 12.03 ▂▇▇▃▁
log_effic_10righttrunc 96 0.90 9.53 0.64 7.79 9.08 9.50 9.98 11.41 ▁▆▇▅▁
plotNrWithin0 0 1.00 2.50 1.71 0.00 1.00 2.50 4.00 5.00 ▇▃▃▃▃
mean(study1_w_timestamp$effic)
[1] 28817.81
sd(study1_w_timestamp$effic)
[1] 67778.16
study1_w_timestamp_type <- study1_w_timestamp %>%
    select(sensi, effic, session, type, plot) %>%
    gather(var, value, sensi, effic)%>%
    mutate(variable = paste(var, type, sep = "_"))%>%
    select(-type, -var)%>%
    group_by(session, plot) %>%
    spread(variable, value) %>%
    ungroup()

skim(study1_w_timestamp_type)
Data summary
Name study1_w_timestamp_type
Number of rows 960
Number of columns 10
_______________________
Column type frequency:
character 2
numeric 8
________________________
Group variables None

Variable type: character

skim_variable n_missing complete_rate min max empty n_unique whitespace
session 0 1 64 64 0 40 0
plot 0 1 8 8 0 24 0

Variable type: numeric

skim_variable n_missing complete_rate mean sd p0 p25 p50 p75 p100 hist
effic_gardneraltman_xaxis 720 0.25 35739.25 99037.01 3547.3 10743.50 17592.75 32584.00 1342595.6 ▇▁▁▁▁
effic_halfeye_xaxis 720 0.25 27179.72 62047.66 2897.0 8788.35 13740.70 23012.73 684967.0 ▇▁▁▁▁
effic_halfeye_yaxis 720 0.25 23954.41 44387.89 2419.0 8960.30 13016.15 23628.10 466834.3 ▇▁▁▁▁
effic_raincloud_yaxis 720 0.25 28397.86 52248.67 2769.0 8910.65 14126.85 25963.05 598871.6 ▇▁▁▁▁
sensi_gardneraltman_xaxis 837 0.13 20619.07 20690.87 4326.0 8835.95 14150.30 24945.10 140178.2 ▇▁▁▁▁
sensi_halfeye_xaxis 846 0.12 23312.30 67326.04 2897.0 7717.35 11111.55 17859.72 684967.0 ▇▁▁▁▁
sensi_halfeye_yaxis 838 0.13 21960.91 56691.29 2419.0 6957.90 9894.00 17751.38 466834.3 ▇▁▁▁▁
sensi_raincloud_yaxis 839 0.13 24843.57 59244.50 2769.0 6906.80 11178.70 21190.10 598871.6 ▇▁▁▁▁
# efficacy
mean(study1_w_timestamp_type$effic_gardneraltman_xaxis, na.rm=TRUE)
[1] 35739.25
sd(study1_w_timestamp_type$effic_gardneraltman_xaxis, na.rm=TRUE)
[1] 99037.01
mean(study1_w_timestamp_type$effic_halfeye_xaxis, na.rm=TRUE)
[1] 27179.72
sd(study1_w_timestamp_type$effic_halfeye_xaxis, na.rm=TRUE)
[1] 62047.66
mean(study1_w_timestamp_type$effic_halfeye_yaxis, na.rm=TRUE)
[1] 23954.41
sd(study1_w_timestamp_type$effic_halfeye_yaxis, na.rm=TRUE)
[1] 44387.89
mean(study1_w_timestamp_type$effic_raincloud_yaxis, na.rm=TRUE)
[1] 28397.86
sd(study1_w_timestamp_type$effic_raincloud_yaxis, na.rm=TRUE)
[1] 52248.67

Accuracy

Distribution of the Dependent Variables

study1_w %>% 
    select(rating_cl, rating_u3, rating_ov) %>% 
    pivot_longer(
        c(rating_cl, rating_u3, rating_ov),
        names_to = "dependent_variable", 
        values_to = "rated_effectsize"
        ) %>% 
    ggplot(., aes(rated_effectsize)) +
    geom_histogram() +
    xlab("Rated effect size") +
    facet_wrap(~dependent_variable, scales = "free_x", 
                labeller = as_labeller(c("rating_cl" = "Abstract metric",
                                         "rating_ov" = "Overlap metric",
                                         "rating_u3" = "Cohen's U3 metric"))) +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white"))

Somewhat disturbing is the first mode in rating_ov. Maybe some users confused overlap and non-overlap? Another artifact seems to be the first mode in rating_u3.

Are there Differences between particpants that indicated a specific school type and those that indicated “others”?

Graphical Overview

study1_w_demo %>% 
    ggplot(., aes(effsize, rating_cl, color = as.factor(schooltype_binary))) +
    geom_jitter() +
    stat_smooth() +
    ylab("Abstract metric") +
    xlab("Plotted effect size") +
    labs(color = "Schooltype") +
    theme_modern_rc()

study1_w_demo%>% 
    ggplot(., aes(effsize, rating_u3, color = as.factor(schooltype_binary))) +
    geom_jitter() +
    stat_smooth() +
    ylab("Cohen's U3 metric") +
    xlab("Plotted effect size") +
    labs(color = "Schooltype") +
    theme_modern_rc()

study1_w_demo %>% 
    ggplot(., aes(abs(effsize), rating_ov, color = as.factor(schooltype_binary))) +
    geom_jitter() +
    stat_smooth(method = "lm") +
    ylab("Overlap metric") +
    xlab("Plotted effect size") +
    labs(color = "Schooltype") +
    theme_modern_rc()

study1_w_demo %>% 
ggplot(., aes(as.factor(schooltype_binary), infor)) +
    geom_jitter()+
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult =1),
                 color = "white") +
    ylab("Perceived Informativity") +
    xlab("School Type 1") +
    theme_modern_rc()

study1_w_demo %>%
ggplot(., aes(as.factor(schooltype_binary), diffi)) +
    geom_jitter()+
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult =1),
                 color = "white") +
    ylab("Perceived Difficulty") +
    xlab("School Type") +
    theme_modern_rc()

study1_w_demo %>%
ggplot(., aes(as.factor(schooltype_binary), value)) +
    geom_jitter()+
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult =1),
                 color = "white") +
    ylab("Perceived Value") +
    xlab("School Type 1=others") +
    theme_modern_rc()

Compute Kendall’s \(\tau\) for rating_cl, rating_u3 and rating_ov

## accuracy U3 
### overall 
study1_w_demo %>% 
    do(tau_u3 = unlist(cor(.$effsize, .$rating_u3, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_u3)
# A tibble: 1 × 1
   tau_u3
    <dbl>
1 -0.0837
### grouped by plot type 
study1_w_demo %>% 
    group_by(type) %>% 
    do(tau_u3 = unlist(cor(.$effsize, .$rating_u3, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_u3)
# A tibble: 4 × 2
  type                 tau_u3
  <chr>                 <dbl>
1 gardneraltman_xaxis -0.0334
2 halfeye_xaxis       -0.143 
3 halfeye_yaxis       -0.0423
4 raincloud_yaxis     -0.118 
### grouped by school type
study1_w_demo %>% 
    group_by(schooltype_binary) %>% 
    do(tau_u3a = unlist(cor(.$effsize, .$rating_u3, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_u3a)
# A tibble: 2 × 2
  schooltype_binary tau_u3a
              <dbl>   <dbl>
1                 0 -0.0649
2                 1 -0.122 
### grouped by school type and plot type
study1_w_demo %>% 
    group_by(schooltype_binary, type) %>% 
    do(tau_u3 = unlist(cor(abs(.$effsize), .$rating_u3, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_u3)
# A tibble: 8 × 3
  schooltype_binary type                tau_u3
              <dbl> <chr>                <dbl>
1                 0 gardneraltman_xaxis 0.105 
2                 0 halfeye_xaxis       0.188 
3                 0 halfeye_yaxis       0.210 
4                 0 raincloud_yaxis     0.0956
5                 1 gardneraltman_xaxis 0.0805
6                 1 halfeye_xaxis       0.131 
7                 1 halfeye_yaxis       0.0252
8                 1 raincloud_yaxis     0.133 
## overlap 

### overall 
study1_w_demo %>% 
    do(tau_ov = unlist(cor(abs(.$effsize), .$rating_ov, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_ov)
# A tibble: 1 × 1
  tau_ov
   <dbl>
1 -0.276
### grouped by plot type 
study1_w_demo %>% 
    group_by(type) %>% 
    do(tau_ov = unlist(cor(abs(.$effsize), .$rating_ov, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_ov)
# A tibble: 4 × 2
  type                tau_ov
  <chr>                <dbl>
1 gardneraltman_xaxis -0.188
2 halfeye_xaxis       -0.228
3 halfeye_yaxis       -0.326
4 raincloud_yaxis     -0.348
### grouped by school type
study1_w_demo %>% 
    group_by(schooltype_binary) %>% 
    do(tau_ov = unlist(cor(abs(.$effsize), .$rating_ov, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_ov)
# A tibble: 2 × 2
  schooltype_binary tau_ov
              <dbl>  <dbl>
1                 0 -0.233
2                 1 -0.345
### grouped by school type and plot type 
study1_w_demo %>% 
    group_by(schooltype_binary, type) %>% 
    do(tau_ov = unlist(cor(abs(.$effsize), .$rating_ov, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_ov)
# A tibble: 8 × 3
  schooltype_binary type                tau_ov
              <dbl> <chr>                <dbl>
1                 0 gardneraltman_xaxis -0.201
2                 0 halfeye_xaxis       -0.105
3                 0 halfeye_yaxis       -0.195
4                 0 raincloud_yaxis     -0.410
5                 1 gardneraltman_xaxis -0.155
6                 1 halfeye_xaxis       -0.434
7                 1 halfeye_yaxis       -0.541
8                 1 raincloud_yaxis     -0.248
## Abstract Metric

### overall
study1_w_demo %>% 
    do(tau_cl = unlist(cor(.$effsize, .$rating_cl, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_cl)
# A tibble: 1 × 1
  tau_cl
   <dbl>
1  0.449
### grouped by plot type
study1_w_demo %>% 
    group_by(type) %>% 
    do(tau_cl = unlist(cor(.$effsize, .$rating_cl, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_cl)
# A tibble: 4 × 2
  type                tau_cl
  <chr>                <dbl>
1 gardneraltman_xaxis  0.373
2 halfeye_xaxis        0.548
3 halfeye_yaxis        0.484
4 raincloud_yaxis      0.415
### grouped by school type
study1_w_demo %>% 
    group_by(schooltype_binary) %>% 
    do(tau_cl = unlist(cor(.$effsize, .$rating_cl, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_cl)
# A tibble: 2 × 2
  schooltype_binary tau_cl
              <dbl>  <dbl>
1                 0  0.373
2                 1  0.592
### grouped by school type and plot type
study1_w_demo %>% 
    group_by(schooltype_binary, type) %>% 
    do(tau_cl = unlist(cor(.$effsize, .$rating_cl, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_cl)
# A tibble: 8 × 3
  schooltype_binary type                tau_cl
              <dbl> <chr>                <dbl>
1                 0 gardneraltman_xaxis  0.379
2                 0 halfeye_xaxis        0.442
3                 0 halfeye_yaxis        0.421
4                 0 raincloud_yaxis      0.280
5                 1 gardneraltman_xaxis  0.371
6                 1 halfeye_xaxis        0.699
7                 1 halfeye_yaxis        0.590
8                 1 raincloud_yaxis      0.701
study1_w_demo_vda <- study1_w_demo %>%
    select(session, plot, rating_ov, rating_cl, rating_u3, schooltype_binary) %>%
    pivot_longer(rating_ov:rating_u3, names_to = "variables", 
                 values_to = "values") %>%
    mutate(var = paste(variables, schooltype_binary, sep = "_")) %>%
    select(-variables, values) %>%
    group_by(session, plot) %>%
    pivot_wider(names_from = "var", values_from = "values")


effsize::VD.A(na.omit(study1_w_demo_vda$rating_cl_1), na.omit(study1_w_demo_vda$rating_cl_0))

Vargha and Delaney A

A estimate: 0.534 (negligible)
effsize::VD.A(na.omit(study1_w_demo_vda$rating_u3_1), na.omit(study1_w_demo_vda$rating_u3_0))

Vargha and Delaney A

A estimate: 0.5272222 (negligible)
effsize::VD.A(na.omit(study1_w_demo_vda$rating_ov_1), na.omit(study1_w_demo_vda$rating_ov_0))

Vargha and Delaney A

A estimate: 0.5908981 (small)

Are there Constant Misconceptions per Persons?

ggplot(study1_w %>% 
    select(rating_cl, rating_u3, rating_ov, effsize, effsize_cl, session) %>% 
    pivot_longer(
        c(rating_cl, rating_u3, rating_ov),
        names_to = "operationalization", 
        values_to = "judged_effectsize"
        ),
    aes(judged_effectsize, as.numeric(as.factor(session)),
        color = session)
        ) +
    geom_jitter(height = 0) +
    xlab("Rated effect size") +
    ylab("Participant ID") +
    facet_wrap(~ operationalization, scales = "free_x", 
                labeller = as_labeller(c("rating_cl" = "Abstract metric",
                                         "rating_ov" = "Overlap metric",
                                         "rating_u3" = "Cohen's U3 metric"))) +
    theme_modern_rc() +
    theme(legend.position = "none",
          strip.text = element_text(color = "white"))

Association of Ratings and Actual Effect Size

study1_w %>% 
    ggplot(., aes(effsize, rating_cl)) +
    geom_jitter() +
    stat_smooth() +
    ylab("Abstract metric") +
    xlab("Plotted effect size") +
    theme_modern_rc()

study1_w %>% 
    ggplot(., aes(effsize, rating_u3, color = rating_u3_missconcept)) +
    geom_jitter() +
    stat_smooth() +
    ylab("Cohen's U3 metric") +
    xlab("Plotted effect size") +
    labs(color = "Misconcept") +
    theme_modern_rc()

study1_w %>% 
    ggplot(., aes(abs(effsize), rating_ov, color = rating_ov_missconcept)) +
    geom_jitter() +
    stat_smooth(method = "lm") +
    ylab("Overlap metric") +
    xlab("Plotted effect size") +
    labs(color = "Misconcept") +
    theme_modern_rc()

The correlations underpin the interpretation of the misconceptions. Therefore, we will look at the intercorrelations of the dependent variables.

Compute Kendall’s \(\tau\) for rating_cl, rating_u3 and rating_ov

study1_w %>% 
    group_by(type) %>% 
    do(tau_cl = unlist(cor(.$effsize, .$rating_cl, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_cl)
# A tibble: 4 × 2
  type                tau_cl
  <chr>                <dbl>
1 gardneraltman_xaxis  0.373
2 halfeye_xaxis        0.548
3 halfeye_yaxis        0.484
4 raincloud_yaxis      0.415
study1_w %>% 
    group_by(rating_ov_missconcept, type) %>% 
    do(tau_ov = unlist(cor(abs(.$effsize), .$rating_ov, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_ov)
# A tibble: 8 × 3
  rating_ov_missconcept type                 tau_ov
  <lgl>                 <chr>                 <dbl>
1 FALSE                 gardneraltman_xaxis -0.355 
2 FALSE                 halfeye_xaxis       -0.384 
3 FALSE                 halfeye_yaxis       -0.478 
4 FALSE                 raincloud_yaxis     -0.523 
5 TRUE                  gardneraltman_xaxis  0.0236
6 TRUE                  halfeye_xaxis        0.119 
7 TRUE                  halfeye_yaxis        0.0205
8 TRUE                  raincloud_yaxis     -0.220 
study1_w %>% 
    group_by(rating_u3_missconcept, type) %>% 
    do(tau_u3 = cor(.$effsize, .$rating_u3, method = "kendall", 
                           use = "pairwise.complete")) %>% 
    unnest(tau_u3)
# A tibble: 8 × 3
  rating_u3_missconcept type                 tau_u3
  <lgl>                 <chr>                 <dbl>
1 FALSE                 gardneraltman_xaxis -0.298 
2 FALSE                 halfeye_xaxis       -0.471 
3 FALSE                 halfeye_yaxis       -0.263 
4 FALSE                 raincloud_yaxis     -0.321 
5 TRUE                  gardneraltman_xaxis  0.119 
6 TRUE                  halfeye_xaxis       -0.153 
7 TRUE                  halfeye_yaxis       -0.108 
8 TRUE                  raincloud_yaxis     -0.0971

How many misconceptualizers can be found among the participants who checked “others” for school type?

u3_misconcept_schooltype <- study1_w_demo %>%
    select(session, rating_u3_missconcept, schooltype_binary) %>%
    unique() 

count(u3_misconcept_schooltype, rating_u3_missconcept, schooltype_binary)
# A tibble: 4 × 3
  rating_u3_missconcept schooltype_binary     n
  <lgl>                             <dbl> <int>
1 FALSE                                 0    11
2 FALSE                                 1     7
3 TRUE                                  0    14
4 TRUE                                  1     8
ov_misconcept_schooltype <- study1_w_demo %>%
    select(session, rating_ov_missconcept, schooltype_binary) %>%
    unique() 

count(ov_misconcept_schooltype, rating_ov_missconcept, schooltype_binary)
# A tibble: 4 × 3
  rating_ov_missconcept schooltype_binary     n
  <lgl>                             <dbl> <int>
1 FALSE                                 0    18
2 FALSE                                 1    13
3 TRUE                                  0     7
4 TRUE                                  1     2

Associations of the Dependent Variables

ggplot(study1_w, 
       aes(rating_cl, rating_u3)) +
    geom_jitter() +
    xlab("Abstract metric") +
    ylab("Cohen's U3 metric") +
    theme_modern_rc() +
ggplot(study1_w, 
       aes(rating_cl, rating_ov)) +
    geom_jitter() +
    xlab("Abstract metric") +
    ylab("Overlap metric") +
    theme_modern_rc() +
ggplot(study1_w, 
       aes(rating_ov, rating_u3)) +
    geom_jitter() +
    xlab("Overlap metric") +
    ylab("Cohen's U3 metric") +
    theme_modern_rc()

Global Under- or Overestimation

study1_w %>% 
    select(diff_cl, diff_u3_filtered, diff_ov_filtered, session, type) %>% 
    gather(dependent_variable, difference_to_true_effsize, 
           diff_cl, diff_u3_filtered, diff_ov_filtered) %>% 
    ggplot(., aes(type, difference_to_true_effsize)) +
    geom_boxplot(alpha = .3) +
    facet_wrap(~dependent_variable, scales = "free_y", 
                labeller = as_labeller(c("diff_cl" = "Abstract metric",
                                         "diff_ov_filtered" = "Overlap metric (filtered)",
                                         "diff_u3_filtered" = "Cohen's U3 metric (filtered)"))) +
    geom_jitter(aes(color = type)) +
    ylab("Difference to true effect size") +
    xlab("Accuracy item") +
    labs(color = "Accuracy item") +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white"),
          axis.text.x = element_blank())

Research Question 1

Accuracy abstract metric

Random Intercept Models With and Without Visualization Type

#| results: hide
# Rating abstract metric
mod0_rating_cl <- brm(rating_cl ~ + (1|session), 
                          data = study1_w,
                          iter = 20000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4)
mod1_rating_cl <- brm(rating_cl ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 20000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4)
mod2_rating_cl <- brm(rating_cl ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
bayes_factor(mod1_rating_cl, mod0_rating_cl)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod1_rating_cl over mod0_rating_cl: 694770923350984417317368128077824.00000
bayes_factor(mod2_rating_cl, mod1_rating_cl)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod2_rating_cl over mod1_rating_cl: 0.00113

Accuracy Cohen’s U3

Random Intercept Models With and Without Visualization Type

#| results: hide
# Rating Cohen's U3
mod0_rating_u3 <- brm(rating_u3 ~ + (1|session), 
                          data = study1_w%>% 
                              filter(rating_u3_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
mod1_rating_u3 <- brm(rating_u3 ~ effsize + (1|session), 
                          data = study1_w%>% 
                              filter(rating_u3_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
mod2_rating_u3 <- brm(rating_u3 ~ type + effsize + (1|session), 
                          data = study1_w %>% 
                              filter(rating_u3_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
bayes_factor(mod1_rating_u3, mod0_rating_u3)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod1_rating_u3 over mod0_rating_u3: 1051568871.23350
bayes_factor(mod2_rating_u3, mod1_rating_u3)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Estimated Bayes factor in favor of mod2_rating_u3 over mod1_rating_u3: 2398.20432

Accuracy overlap

Random Intercept Models With and Without Visualization Type

#| results: hide
# Rating Overlap
mod0_rating_ov <- brm(rating_ov ~ + (1|session), 
                          data = study1_w%>% 
                              filter(rating_ov_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
mod1_rating_ov <- brm(rating_ov ~ effsize + (1|session), 
                          data = study1_w%>% 
                              filter(rating_ov_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
mod2_rating_ov <- brm(rating_ov ~ type + effsize + (1|session), 
                          data = study1_w %>% 
                              filter(rating_ov_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
bayes_factor(mod1_rating_ov, mod0_rating_ov)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod1_rating_ov over mod0_rating_ov: 4.80568
bayes_factor(mod2_rating_ov, mod1_rating_ov)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod2_rating_ov over mod1_rating_ov: 538.28196

Perceived Difficulty

Graphichal Overview

study1_w %>% 
    ggplot(aes(type, diffi)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Difficulty",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

Random Intercept Models With and Without Visualization Type

# Perceived Difficulty
mod0_diffi <- brm(diffi ~ + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
mod1_diffi <- brm(diffi ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
mod2_diffi <- brm(diffi ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
bayes_factor(mod1_diffi, mod0_diffi)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Estimated Bayes factor in favor of mod1_diffi over mod0_diffi: 0.39498
bayes_factor(mod2_diffi, mod1_diffi)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod2_diffi over mod1_diffi: 4874251106763547.00000

Perceived Informativity

Graphical Overview

study1_w %>% 
    ggplot(aes(type, infor)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Informativity",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

Random Intercept Models With and Without Visualization Type

# Perceived Informativity
mod0_infor <- brm(infor ~ + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
mod1_infor <- brm(infor ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
mod2_infor <- brm(infor ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
bayes_factor(mod1_infor, mod0_infor)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod1_infor over mod0_infor: 0.25868
bayes_factor(mod2_infor, mod1_infor)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod2_infor over mod1_infor: 727215.56159

Perceived Value

Graphical Overview

study1_w %>% 
    ggplot(aes(type, value)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Perceived Value",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

Random Intercept Models With and Without Visualization Type

# Perceived Value 
mod0_value <- brm(value ~ + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
mod1_value <- brm(value ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
mod2_value <- brm(value ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
bayes_factor(mod1_value, mod0_value)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod1_value over mod0_value: 0.21894
bayes_factor(mod2_value, mod1_value)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod2_value over mod1_value: 84034532.38619

Table all random intercept models

sjPlot::tab_model(
    mod0_rating_cl, mod0_rating_u3, mod0_rating_ov,
    mod0_diffi, mod0_infor, mod0_value,
    mod1_rating_cl, mod1_rating_u3, mod1_rating_ov,
    mod1_diffi, mod1_infor, mod1_value,
    mod2_rating_cl, mod2_rating_u3, mod2_rating_ov,
    mod2_diffi, mod2_infor, mod2_value
    )
  rating_cl rating_u3 rating_ov diffi infor value rating_cl rating_u3 rating_ov diffi infor value rating_cl rating_u3 rating_ov diffi infor value
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept 0.02 -0.02 – 0.05 43.09 38.48 – 47.78 82.69 80.18 – 85.26 4.05 3.68 – 4.40 4.11 3.81 – 4.40 4.13 3.79 – 4.47 0.01 -0.02 – 0.05 43.34 38.71 – 48.00 82.70 80.19 – 85.22 4.05 3.68 – 4.42 4.11 3.81 – 4.40 4.13 3.80 – 4.47 0.01 -0.05 – 0.07 42.71 36.85 – 48.44 82.30 78.50 – 86.23 3.59 3.18 – 3.97 3.80 3.48 – 4.14 3.79 3.42 – 4.13
effsize 0.37 0.32 – 0.42 -11.79 -15.33 – -8.19 -1.04 -4.02 – 2.00 -0.09 -0.24 – 0.06 -0.06 -0.19 – 0.07 -0.05 -0.17 – 0.08 0.37 0.32 – 0.42 -11.62 -15.16 – -8.10 -1.09 -4.11 – 1.92 -0.09 -0.23 – 0.05 -0.06 -0.19 – 0.07 -0.05 -0.17 – 0.07
typehalfeye_xaxis -0.02 -0.10 – 0.06 0.54 -5.07 – 6.19 2.46 -2.22 – 7.16 0.65 0.42 – 0.87 0.39 0.19 – 0.59 0.51 0.32 – 0.70
typehalfeye_yaxis 0.01 -0.08 – 0.09 4.10 -1.62 – 9.81 -0.98 -5.78 – 3.77 0.97 0.74 – 1.19 0.61 0.40 – 0.81 0.58 0.39 – 0.77
typeraincloud_yaxis 0.01 -0.08 – 0.09 -1.85 -7.62 – 3.90 -0.14 -4.90 – 4.76 0.24 0.02 – 0.47 0.24 0.04 – 0.44 0.23 0.04 – 0.42
Random Effects
σ2 0.15 259.13 254.81 1.72 1.32 1.17 0.11 214.02 255.24 1.72 1.32 1.17 0.11 213.00 256.19 1.58 1.27 1.11
τ00 0.00 session 75.29 session 29.95 session 1.31 session 0.87 session 1.15 session 0.00 session 82.64 session 29.80 session 1.31 session 0.87 session 1.14 session 0.00 session 79.60 session 29.09 session 1.32 session 0.87 session 1.14 session
ICC 0.01 0.23 0.11 0.43 0.40 0.50 0.02 0.28 0.10 0.43 0.40 0.49 0.02 0.27 0.10 0.46 0.41 0.51
N 40 session 18 session 31 session 40 session 40 session 40 session 40 session 18 session 31 session 40 session 40 session 40 session 40 session 18 session 31 session 40 session 40 session 40 session
Observations 480 216 372 960 960 960 480 216 372 960 960 960 480 216 372 960 960 960
Marginal R2 / Conditional R2 0.000 / 0.006 0.000 / 0.198 0.000 / 0.098 0.000 / 0.413 0.000 / 0.376 0.000 / 0.475 0.279 / 0.291 0.134 / 0.343 0.002 / 0.101 0.001 / 0.414 0.001 / 0.377 0.000 / 0.476 0.282 / 0.294 0.154 / 0.359 0.015 / 0.112 0.050 / 0.463 0.025 / 0.401 0.026 / 0.501

Sensitivity

Graphical Overview

ggplot(study1_w %>% 
           select(sensi_ordinal, type) %>% 
           na.omit(), aes(sensi_ordinal, 
                          color = sensi_ordinal,
                          fill = sensi_ordinal)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Sensitivity", "all participants")

ggplot(study1_w %>% 
           select(sensi_ordinal, type, session) %>% 
           na.omit() %>% 
           filter(!session %in% ov_misconceptualizers & 
                      !session %in% u3_misconceptualizers), aes(sensi_ordinal, 
                          color = sensi_ordinal,
                          fill = sensi_ordinal)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Sensitivity", "without missconceptualizers")

ggplot(study1_w %>% 
           select(sensi_correct, type, session) %>%
           dplyr::filter(!is.na(sensi_correct)),
       aes(sensi_correct, 
           color = sensi_correct,
           fill = sensi_correct)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white"),
          axis.text.x = element_text(angle = 45, hjust = 1)) +
    labs(color = "", fill = "") +
    ggtitle("Binary Sensitivity", "without wrong decisions")

ggplot(study1_w %>% 
           select(sensi_binary_filtered, type, session),
       aes(sensi_binary_filtered, 
           color = as.factor(sensi_binary_filtered),
           fill = as.factor(sensi_binary_filtered))) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    labs(color = "", fill = "") +
    ggtitle("Binary Sensitivity", "without wrong decisions (0=equal, 1=right direction)")

Logistic Regressions With and Without Visualization Type

Fitting a series of logistic regressions with non-informative priors

#| results: hide

logreg_mod0 <- brm(
    sensi_binary_filtered ~ +(1 | session),
    family = bernoulli(link = "logit"),
    data = study1_w,
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4
    
)

logreg_mod1 <-
    brm(
        sensi_binary_filtered ~ effsize_abs + (1 | session),
        family = bernoulli(link = "logit"),
        data = study1_w,
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4
        
    )

logreg_mod2 <-
    brm(
        sensi_binary_filtered ~ effsize_abs + type + (1 | session),
        family = bernoulli(link = "logit"),
        data = study1_w,
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4
        
    )
bayes_factor(logreg_mod1, logreg_mod0)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 7
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of logreg_mod1 over logreg_mod0: 435182901565522720645523177472.00000
bayes_factor(logreg_mod2, logreg_mod1)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of logreg_mod2 over logreg_mod1: 15.67848
sjPlot::tab_model(logreg_mod0, logreg_mod1, logreg_mod2, show.icc = TRUE)
  sensi_binary_filtered sensi_binary_filtered sensi_binary_filtered
Predictors Odds Ratios CI (95%) Odds Ratios CI (95%) Odds Ratios CI (95%)
Intercept 3.08 1.56 – 6.31 0.11 0.03 – 0.34 0.08 0.02 – 0.34
effsize_abs 2852.08 509.02 – 19973.65 3497.57 574.60 – 26363.85
typehalfeye_xaxis 1.92 0.73 – 5.06
typehalfeye_yaxis 1.84 0.73 – 4.84
typeraincloud_yaxis 0.77 0.31 – 1.90
Random Effects
σ2 3.29 3.29 3.29
τ00 3.85 session 10.23 session 11.27 session
ICC 0.54 0.76 0.77
N 40 session 40 session 40 session
Observations 453 453 453
Marginal R2 / Conditional R2 0.000 / 0.318 0.299 / 0.597 0.306 / 0.605
pp_check(logreg_mod2) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

Efficiency

Visualisation

Raw data

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,100000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

5% Percent Truncated

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic_05righttrunc)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,85000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="5% Truncated Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

10% Percent Truncated

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic_10righttrunc)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,85000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="10% Truncated Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

log() Transformed

ggplot(study1_w_timestamp, 
       aes(as.factor(plotNrWithin), 
           log(effic_05righttrunc))) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="log Transformed Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

Modeling with {brms}

Efficiency for First Plot

#| error: false
#| warning: false
#| results: hide

study1_w_timestamp_effsize <- study1_w_timestamp %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")

plot01_mod0 <-
    brm(log_effic_05righttrunc ~ 1 + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot01_mod0) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

#| error: false
#| warning: false
#| results: hide
plot01_mod1 <-
    brm(log_effic_05righttrunc~ 1 + effsize + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot01_mod1) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

#| error: false
#| warning: false
#| results: hide
plot01_mod2 <-
    brm(log_effic_05righttrunc ~ 1 + effsize + type + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot01_mod2) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

#| error: false
#| warning: false
#| results: hide
sjPlot::tab_model(plot01_mod0, plot01_mod1, plot01_mod2)
  log_effic_05righttrunc log_effic_05righttrunc log_effic_05righttrunc
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept 10.13 9.96 – 10.28 10.13 9.97 – 10.28 10.50 10.33 – 10.66
effsize -0.00 -0.03 – 0.03 0.00 -0.03 – 0.03
typehalfeye_xaxis -0.43 -0.47 – -0.39
typehalfeye_yaxis -0.54 -0.58 – -0.49
typeraincloud_yaxis -0.50 -0.55 – -0.46
Random Effects
σ2 0.25 0.25 0.20
τ00 0.24 session 0.25 session 0.25 session
ICC 0.49 0.49 0.55
N 40 session 40 session 40 session
Observations 3648 3648 3648
Marginal R2 / Conditional R2 0.000 / 0.458 0.000 / 0.458 0.100 / 0.558
bayes_factor(plot01_mod1, plot01_mod0)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Estimated Bayes factor in favor of plot01_mod1 over plot01_mod0: 0.03892
bayes_factor(plot01_mod2, plot01_mod1)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 7
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of plot01_mod2 over plot01_mod1: 1770776011793288578099518651780334949935988681027090403823789893788615439226156721325265870403861352189977128553873231777644561722392387052732500347386331136.00000

Efficiency for Last Three Plots

#| error: false
#| warning: false
#| results: hide
plot0306_mod0 <-
    brm(log_effic_05righttrunc ~ 1 + plotNrWithin_factor + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin >= 4),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot0306_mod0) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

#| error: false
#| warning: false
#| results: hide
plot0306_mod1 <-
    brm(log_effic_05righttrunc ~ 1 + plotNrWithin_factor + effsize +
            (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin >= 4),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4)

pp_check(plot0306_mod1) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

#| error: false
#| warning: false
#| results: hide
plot0306_mod2 <-
    brm(log_effic_05righttrunc ~ 1 + plotNrWithin_factor + effsize +
            type + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin >= 4),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4)

pp_check(plot0306_mod2) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

sjPlot::tab_model(plot0306_mod0, plot0306_mod1, plot0306_mod2)
  log_effic_05righttrunc log_effic_05righttrunc log_effic_05righttrunc
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept 9.48 9.35 – 9.61 9.48 9.35 – 9.62 9.59 9.46 – 9.72
plotNrWithin_factor:
plotNrWithin_factor5
-0.01 -0.03 – 0.02 -0.01 -0.03 – 0.01 -0.01 -0.03 – 0.01
plotNrWithin_factor:
plotNrWithin_factor6
-0.19 -0.21 – -0.17 -0.19 -0.21 – -0.17 -0.19 -0.21 – -0.17
effsize 0.00 -0.02 – 0.02 0.00 -0.02 – 0.02
typehalfeye_xaxis -0.13 -0.16 – -0.11
typehalfeye_yaxis -0.21 -0.23 – -0.18
typeraincloud_yaxis -0.06 -0.09 – -0.04
Random Effects
σ2 0.24 0.24 0.24
τ00 0.17 session 0.17 session 0.17 session
ICC 0.41 0.41 0.42
N 40 session 40 session 40 session
Observations 10944 10944 10944
Marginal R2 / Conditional R2 0.019 / 0.406 0.019 / 0.406 0.034 / 0.421
bayes_factor(plot0306_mod1, plot0306_mod0)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of plot0306_mod1 over plot0306_mod0: 0.02335
bayes_factor(plot0306_mod2, plot0306_mod1)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Estimated Bayes factor in favor of plot0306_mod2 over plot0306_mod1: 18416094205724295475395080047976396374990194327754375168.00000

Research Question 2

Contrast Effects

Graphical Overview

#| results: hide

# preparing multilevel model with brms for graphical overview

# rating u3  
mod_plot_ratingu3 <- brm(
    scale(rating_u3) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 10000
    
) 

plot_ratingu3 <- mod_plot_ratingu3 %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean
             #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
    stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_text(hjust = 0)) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle("Accuracy in Cohen's U3 Metric")
plot_ratingu3

#| results: hide
# rating ov
mod_plot_ratingov <- brm (
    scale(rating_ov) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 10000
    
)

plot_ratingov <- mod_plot_ratingov %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean
             #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
    stat_halfeye() + 
    theme_ipsum_rc()+
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Accuracy in Overlap Metric')
plot_ratingov

#| results: hide
# difficulty
mod_plot_diffi <- brm (
    scale(diffi) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 10000
    
)

plot_diffi <- mod_plot_diffi %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean
             #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
    stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Perceived Task Difficulty')
plot_diffi

#| results: hide
# informativity
mod_plot_infor <- brm (
    scale(infor) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 10000
    
)

plot_infor <- mod_plot_infor %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
  stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_text(hjust = 0)) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Perceived Informativity')
plot_infor

#| results: hide
# value 

mod_plot_value <- brm (
    scale(value) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 10000
    
)

plot_value <- mod_plot_value %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
  stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Perceived Value')
plot_value

# efficiency for the first plot

mod_plot_efficiency1 <-
    brm(
        scale(effic) ~ (1 | type),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        control = list(adapt_delta = .999),
        cores = 4
        
    )
 

plot_efficiency1 <- mod_plot_efficiency1 %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
  stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Efficiency for the First Plot')
plot_efficiency1

# plot_contrasteffects <- plot_ratingu3 + plot_ratingov + plot_diffi + plot_infor + plot_value + plot_efficiency1 + 
#     plot_layout(ncol=3)

# ggsave("www/plot_contrasteffects.png", 
#        plot_contrasteffects,
#        width = 25*.55,
#        height = 16*.55)

Model Contrast Effects with {brms}

# create dummy variables to compare visualization types for those measures that showed evidence for difference (accuracy U3, accuracy overlap, perceived task difficulty, perceived informativity, perceived value, efficiency)

## Gardner Altman Plot vs. Halfeye xaxis
gardneraltman_vs_halfeye_x <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_x = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_xaxis") %>%
    ungroup()

gardneraltman_vs_halfeye_x_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_x = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_xaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")
#| results: hide
mod_accu3_gardneraltman_halfeye_x <- 
    brm(scale(rating_u3) ~ dummy_gardneraltman_halfeye_x + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_x%>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

mod_accov_gardneraltman_halfeye_x <- 
    brm(scale(rating_ov) ~ dummy_gardneraltman_halfeye_x + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_x%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
#| results: hide
mod_diffi_gardneraltman_halfeye_x <- 
    brm(scale(diffi) ~ dummy_gardneraltman_halfeye_x + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

mod_infor_gardneraltman_halfeye_x <- 
    brm(scale(infor) ~ dummy_gardneraltman_halfeye_x + effsize
        + (1|session), 
                          data = gardneraltman_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
mod_value_gardneraltman_halfeye_x <- 
    brm(scale(value) ~ dummy_gardneraltman_halfeye_x + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE))

mod_effic01_gardneraltman_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_halfeye_x + 
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_x_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
mod_effic0306_gardneraltman_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_halfeye_x +
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_x_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

                  
## Gardner Altman Plot vs. Halfeye yaxis
gardneraltman_vs_halfeye_y <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_y = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_yaxis") %>%
    ungroup()
gardneraltman_vs_halfeye_y_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_y = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_yaxis") %>%
    ungroup()  %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")


mod_accu3_gardneraltman_halfeye_y <- 
    brm(scale(rating_u3) ~ dummy_gardneraltman_halfeye_y + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_y%>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
mod_accov_gardneraltman_halfeye_y <- 
    brm(scale(rating_ov) ~ dummy_gardneraltman_halfeye_y + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_y%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_diffi_gardneraltman_halfeye_y <- 
    brm(scale(diffi) ~ dummy_gardneraltman_halfeye_y + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
mod_infor_gardneraltman_halfeye_y <- 
    brm(scale(infor) ~ dummy_gardneraltman_halfeye_y + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_value_gardneraltman_halfeye_y <- 
    brm(scale(value) ~ dummy_gardneraltman_halfeye_y + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
mod_effic01_gardneraltman_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_halfeye_y +
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_y_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_effic0306_gardneraltman_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_halfeye_y + 
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_y_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
## Raincloud Plot vs. Halfeye x axis
raincloud_vs_halfeye_x <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_x = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_xaxis" | type == "raincloud_yaxis") %>%
    ungroup()

raincloud_vs_halfeye_x_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_x = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "raincloud_yaxis" | type == "halfeye_xaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")
mod_accu3_raincloud_halfeye_x <- 
    brm(scale(rating_u3) ~ dummy_raincloud_halfeye_x + effsize + (1|session), 
                          data = raincloud_vs_halfeye_x %>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_accov_raincloud_halfeye_x <- 
    brm(scale(rating_ov) ~ dummy_raincloud_halfeye_x + effsize + (1|session), 
                          data = raincloud_vs_halfeye_x %>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
mod_diffi_raincloud_halfeye_x <- 
    brm(scale(diffi) ~ dummy_raincloud_halfeye_x + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_infor_raincloud_halfeye_x <- 
    brm(scale(infor) ~ dummy_raincloud_halfeye_x +
        effsize + (1|session), 
                          data = raincloud_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
mod_value_raincloud_halfeye_x <- 
    brm(scale(value) ~ dummy_raincloud_halfeye_x + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_effic01_raincloud_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_x + 
            effsize + (1|session),
    data = raincloud_vs_halfeye_x_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
mod_effic0306_raincloud_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_x +
            effsize + (1|session),
    data = raincloud_vs_halfeye_x_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
##  Raincloud Plot vs.Halfeye yaxis
raincloud_vs_halfeye_y <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_y = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_yaxis" | type == "raincloud_yaxis") %>%
    ungroup()

raincloud_vs_halfeye_y_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_y = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "raincloud_yaxis" | type == "halfeye_yaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")
mod_accu3_raincloud_halfeye_y <- 
    brm(scale(rating_u3) ~ dummy_raincloud_halfeye_y + effsize + (1|session), 
                          data = raincloud_vs_halfeye_y %>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_accov_raincloud_halfeye_y <- 
    brm(scale(rating_ov) ~ dummy_raincloud_halfeye_y + effsize + (1|session), 
                          data = raincloud_vs_halfeye_y %>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
mod_diffi_raincloud_halfeye_y <- 
    brm(scale(diffi) ~ dummy_raincloud_halfeye_y + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_infor_raincloud_halfeye_y <- 
    brm(scale(infor) ~ dummy_raincloud_halfeye_y + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
mod_value_raincloud_halfeye_y <- 
    brm(scale(value) ~ dummy_raincloud_halfeye_y + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_effic01_raincloud_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_y +
            effsize + (1|session),
    data = raincloud_vs_halfeye_y_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

mod_effic0306_raincloud_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_y +
            effsize + (1|session),
    data = raincloud_vs_halfeye_y_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
sjPlot::tab_model(mod_accu3_gardneraltman_halfeye_x, 
        mod_accov_gardneraltman_halfeye_x, mod_diffi_gardneraltman_halfeye_x, 
        mod_infor_gardneraltman_halfeye_x, mod_value_gardneraltman_halfeye_x,
        mod_effic01_gardneraltman_halfeye_x, mod_effic0306_gardneraltman_halfeye_x,
        mod_accu3_gardneraltman_halfeye_y, mod_accov_gardneraltman_halfeye_y, 
        mod_diffi_gardneraltman_halfeye_y, mod_infor_gardneraltman_halfeye_y, 
        mod_value_gardneraltman_halfeye_y,
        mod_effic01_gardneraltman_halfeye_y, mod_effic0306_gardneraltman_halfeye_y,
        mod_accov_raincloud_halfeye_x, mod_diffi_raincloud_halfeye_x, 
        mod_infor_raincloud_halfeye_x, mod_value_raincloud_halfeye_x,
        mod_effic01_raincloud_halfeye_x, mod_effic0306_raincloud_halfeye_x,
        mod_accu3_raincloud_halfeye_y, mod_accov_raincloud_halfeye_y, 
        mod_diffi_raincloud_halfeye_y, mod_infor_raincloud_halfeye_y, 
        mod_value_raincloud_halfeye_y,
        mod_effic01_raincloud_halfeye_y, mod_effic0306_raincloud_halfeye_y)
  scale(rating_u3) scale(rating_ov) scale(diffi) scale(infor) scale(value) scale(log_effic_05righttrunc) scale(log_effic_05righttrunc) scale(rating_u3) scale(rating_ov) scale(diffi) scale(infor) scale(value) scale(log_effic_05righttrunc) scale(log_effic_05righttrunc) scale(rating_ov) scale(diffi) scale(infor) scale(value) scale(log_effic_05righttrunc) scale(log_effic_05righttrunc) scale(rating_u3) scale(rating_ov) scale(diffi) scale(infor) scale(value) scale(log_effic_05righttrunc) scale(log_effic_05righttrunc)
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept -0.01 -0.34 – 0.34 -0.11 -0.34 – 0.12 -0.19 -0.44 – 0.07 -0.13 -0.39 – 0.12 -0.17 -0.43 – 0.09 0.33 0.09 – 0.58 0.10 -0.10 – 0.31 -0.17 -0.51 – 0.17 0.01 -0.24 – 0.26 -0.27 -0.51 – -0.03 -0.20 -0.43 – 0.04 -0.19 -0.41 – 0.05 0.43 0.17 – 0.67 0.17 -0.03 – 0.37 -0.11 -0.33 – 0.09 -0.13 -0.36 – 0.12 -0.05 -0.27 – 0.17 -0.09 -0.34 – 0.16 -0.05 -0.35 – 0.23 0.07 -0.16 – 0.29 -0.16 -0.47 – 0.15 0.03 -0.19 – 0.27 -0.23 -0.45 – 0.01 -0.12 -0.35 – 0.10 -0.12 -0.36 – 0.12 0.05 -0.22 – 0.33 0.14 -0.09 – 0.37
dummy_gardneraltman_halfeye_x 0.04 -0.28 – 0.35 0.18 -0.11 – 0.46 0.37 0.25 – 0.49 0.26 0.14 – 0.39 0.35 0.24 – 0.46 -0.69 -0.75 – -0.64 -0.20 -0.24 – -0.16
effsize -0.73 -1.01 – -0.44 0.04 -0.22 – 0.30 -0.04 -0.15 – 0.06 -0.02 -0.13 – 0.09 -0.05 -0.16 – 0.05 0.00 -0.05 – 0.05 0.00 -0.04 – 0.04 -0.50 -0.80 – -0.18 -0.07 -0.34 – 0.19 -0.04 -0.14 – 0.07 -0.03 -0.14 – 0.09 -0.02 -0.12 – 0.09 -0.00 -0.04 – 0.04 0.00 -0.04 – 0.04 -0.07 -0.32 – 0.20 -0.07 -0.19 – 0.04 -0.06 -0.18 – 0.07 -0.05 -0.16 – 0.08 -0.00 -0.05 – 0.05 -0.00 -0.04 – 0.03 -0.53 -0.85 – -0.22 -0.15 -0.41 – 0.10 -0.06 -0.18 – 0.05 -0.06 -0.18 – 0.06 -0.01 -0.13 – 0.10 -0.00 -0.04 – 0.04 -0.00 -0.03 – 0.03
dummy_gardneraltman_halfeye_y 0.29 -0.06 – 0.66 -0.03 -0.30 – 0.24 0.54 0.42 – 0.66 0.40 0.27 – 0.54 0.39 0.27 – 0.51 -0.79 -0.83 – -0.74 -0.32 -0.37 – -0.28
dummy_raincloud_halfeye_x 0.21 -0.06 – 0.49 0.25 0.11 – 0.38 0.10 -0.04 – 0.24 0.19 0.06 – 0.31 0.15 0.10 – 0.20 -0.12 -0.16 – -0.08
dummy_raincloud_halfeye_y 0.35 0.01 – 0.70 -0.05 -0.35 – 0.24 0.43 0.30 – 0.56 0.25 0.11 – 0.40 0.24 0.11 – 0.37 0.00 -0.05 – 0.05 -0.23 -0.27 – -0.19
Random Effects
σ2 0.64 0.95 0.43 0.46 0.40 0.35 0.58 0.78 0.83 0.47 0.55 0.47 0.25 0.58 0.98 0.55 0.62 0.50 0.31 0.54 0.78 0.93 0.54 0.60 0.53 0.26 0.51
τ00 0.27 session 0.07 session 0.59 session 0.57 session 0.63 session 0.60 session 0.44 session 0.21 session 0.22 session 0.51 session 0.45 session 0.52 session 0.72 session 0.42 session 0.04 session 0.48 session 0.42 session 0.55 session 0.78 session 0.50 session 0.15 session 0.10 session 0.46 session 0.42 session 0.50 session 0.86 session 0.52 session
ICC 0.29 0.07 0.58 0.55 0.62 0.63 0.43 0.21 0.21 0.52 0.45 0.53 0.74 0.42 0.04 0.47 0.40 0.53 0.71 0.48 0.16 0.09 0.46 0.41 0.48 0.77 0.51
N 18 session 31 session 40 session 40 session 40 session 39 session 40 session 18 session 31 session 40 session 40 session 40 session 40 session 40 session 31 session 40 session 40 session 40 session 39 session 40 session 18 session 31 session 40 session 40 session 40 session 40 session 40 session
Observations 109 185 480 480 480 1824 5472 104 177 480 480 480 1824 5472 195 480 480 480 1824 5472 107 187 480 480 480 1824 5472
Marginal R2 / Conditional R2 0.169 / 0.393 0.014 / 0.081 0.036 / 0.579 0.019 / 0.543 0.032 / 0.608 0.118 / 0.653 0.010 / 0.418 0.106 / 0.269 0.008 / 0.205 0.074 / 0.540 0.042 / 0.452 0.040 / 0.532 0.151 / 0.752 0.027 / 0.419 0.018 / 0.055 0.018 / 0.460 0.005 / 0.385 0.011 / 0.510 0.006 / 0.688 0.004 / 0.461 0.130 / 0.262 0.013 / 0.109 0.050 / 0.467 0.018 / 0.408 0.015 / 0.472 0.000 / 0.744 0.013 / 0.487
## Gardner Altman Plot vs. Raincloud Plot
gardneraltman_vs_raincloud <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_raincloud = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "raincloud_yaxis") %>%
    ungroup()

gardneraltman_vs_raincloud_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_raincloud = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "raincloud_yaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")
mod_accu3_gardneraltman_raincloud <- 
    brm(scale(rating_u3) ~ dummy_gardneraltman_raincloud + effsize + (1|session), 
                          data = gardneraltman_vs_raincloud%>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_accov_gardneraltman_raincloud <- 
    brm(scale(rating_ov) ~ dummy_gardneraltman_raincloud + effsize + (1|session), 
                          data = gardneraltman_vs_raincloud%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
mod_diffi_gardneraltman_raincloud <- 
    brm(scale(diffi) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session), 
                          data = gardneraltman_vs_raincloud,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_infor_gardneraltman_raincloud <- 
    brm(scale(infor) ~ dummy_gardneraltman_raincloud +
            effsize + (1|session), 
        data = gardneraltman_vs_raincloud,
        save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
mod_value_gardneraltman_raincloud <- 
    brm(scale(value) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session), 
                          data = gardneraltman_vs_raincloud,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)

mod_effic01_gardneraltman_raincloud <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session),
    data = gardneraltman_vs_raincloud_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

mod_effic0306_gardneraltman_raincloud <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session),
    data = gardneraltman_vs_raincloud_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
## Halfeye xaxis vs. Halfeye yaxis

halfeye_x_vs_halfeye_y <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_halfeye_x_halfeye_y = 
               case_when(type == "halfeye_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_xaxis" | type == "halfeye_yaxis") %>%
    ungroup()

halfeye_x_vs_halfeye_y_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_halfeye_x_halfeye_y = 
               case_when(type == "halfeye_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_xaxis" | type == "halfeye_yaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")
mod_accu3_halfeye_x_halfeye_y <- 
    brm(scale(rating_u3) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y %>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
        
    silent = 2,
    refresh = 0,
    cores = 4)

mod_accov_halfeye_x_halfeye_y <- 
    brm(scale(rating_ov) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
        
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 20000)
mod_diffi_halfeye_x_halfeye_y <- 
    brm(scale(diffi) ~ dummy_halfeye_x_halfeye_y +
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE))

mod_infor_halfeye_x_halfeye_y <- 
    brm(scale(infor) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
mod_value_halfeye_x_halfeye_y <- 
    brm(scale(value) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)

mod_effic01_halfeye_x_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session),
    data = halfeye_x_vs_halfeye_y_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    iter = 10000)
mod_effic0306_halfeye_x_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session),
    data = halfeye_x_vs_halfeye_y_effic%>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
sjPlot::tab_model(mod_accu3_halfeye_x_halfeye_y, mod_accov_halfeye_x_halfeye_y, 
                  mod_diffi_halfeye_x_halfeye_y, mod_infor_halfeye_x_halfeye_y, 
                  mod_value_halfeye_x_halfeye_y,
                  mod_effic01_halfeye_x_halfeye_y)
  scale(rating_u3) scale(rating_ov) scale(diffi) scale(infor) scale(value) scale(log_effic_05righttrunc)
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept -0.09 -0.41 – 0.24 0.11 -0.10 – 0.32 -0.10 -0.35 – 0.15 -0.07 -0.34 – 0.17 -0.03 -0.29 – 0.24 0.11 -0.17 – 0.38
dummy_halfeye_x_halfeye_y 0.16 -0.16 – 0.50 -0.23 -0.51 – 0.06 0.20 0.08 – 0.33 0.16 0.04 – 0.28 0.05 -0.06 – 0.16 -0.14 -0.20 – -0.08
effsize -0.67 -0.97 – -0.38 -0.04 -0.31 – 0.22 -0.01 -0.12 – 0.10 0.04 -0.07 – 0.15 0.05 -0.05 – 0.16 -0.00 -0.05 – 0.05
Random Effects
σ2 0.71 0.98 0.48 0.47 0.41 0.37
τ00 0.25 session 0.03 session 0.55 session 0.58 session 0.65 session 0.72 session
ICC 0.26 0.03 0.53 0.55 0.61 0.66
N 18 session 31 session 40 session 40 session 40 session 39 session
Observations 109 194 480 480 480 1824
Marginal R2 / Conditional R2 0.148 / 0.334 0.019 / 0.052 0.012 / 0.521 0.008 / 0.538 0.002 / 0.598 0.005 / 0.626
sjPlot::tab_model(mod_effic0306_halfeye_x_halfeye_y,
                  mod_accu3_gardneraltman_raincloud, 
                  mod_accov_gardneraltman_raincloud, 
                  mod_diffi_gardneraltman_raincloud)
  scale(log_effic_05righttrunc) scale(rating_u3) scale(rating_ov) scale(diffi)
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept 0.08 -0.17 – 0.30 0.12 -0.21 – 0.45 0.01 -0.23 – 0.27 -0.06 -0.29 – 0.16
dummy_halfeye_x_halfeye_y -0.12 -0.16 – -0.09
effsize 0.00 -0.03 – 0.03 -0.69 -0.98 – -0.40 -0.10 -0.35 – 0.14 -0.09 -0.22 – 0.03
dummy_gardneraltman_raincloud -0.18 -0.48 – 0.15 -0.01 -0.30 – 0.28 0.14 -0.00 – 0.27
Random Effects
σ2 0.52 0.69 0.89 0.59
τ00 0.52 session 0.24 session 0.14 session 0.45 session
ICC 0.50 0.26 0.14 0.43
N 40 session 18 session 31 session 40 session
Observations 5472 107 178 480
Marginal R2 / Conditional R2 0.004 / 0.482 0.159 / 0.353 0.010 / 0.148 0.009 / 0.420
sjPlot::tab_model(mod_infor_gardneraltman_raincloud, 
                  mod_value_gardneraltman_raincloud,
                  mod_effic01_gardneraltman_raincloud, 
                  mod_effic0306_gardneraltman_raincloud)
  scale(infor) scale(value) scale(log_effic_05righttrunc) scale(log_effic_05righttrunc)
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept -0.09 -0.32 – 0.14 -0.06 -0.31 – 0.18 0.41 0.16 – 0.68 0.05 -0.18 – 0.27
dummy_gardneraltman_raincloud 0.16 0.01 – 0.30 0.15 0.02 – 0.28 -0.77 -0.81 – -0.72 -0.10 -0.14 – -0.06
effsize -0.11 -0.24 – 0.01 -0.11 -0.22 – 0.00 0.00 -0.04 – 0.04 0.00 -0.04 – 0.04
Random Effects
σ2 0.60 0.50 0.25 0.57
τ00 0.44 session 0.53 session 0.70 session 0.45 session
ICC 0.42 0.52 0.73 0.44
N 40 session 40 session 39 session 40 session
Observations 480 480 1824 5472
Marginal R2 / Conditional R2 0.012 / 0.407 0.010 / 0.505 0.144 / 0.747 0.003 / 0.426

References

Arslan, Ruben C., Matthias P. Walther, and Cyril S. Tata. 2020. “Formr: A Study Framework Allowing for Automated Feedback Generation and Complex Longitudinal Experience-Sampling Studies Using R.” Behavior Research Methods 52 (1): 376–87. https://doi.org/10.3758/s13428-019-01236-y.